package cn.neu.leon.util;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringEscapeUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * 通过url在线爬取微博，并进行分词，情感词典匹配,得到情感百分比的Map
 * @author leon
 *
 */
public class WeiboCrawler {
	private String title;					//首页标题
	private String screenName;              //微博昵称
	private int statusesCount;             //发布微博数
	private String statusesCountStr;        //发布微博数
	private int pageCount;					  //微博页数
	private String followCount;             //关注数
	private String fansCount;           //粉丝数
	private String userName;              //通过url获取到微博用户名或者用户id
	private boolean idExist = true;             //用户是否存在
	private String genderAndRegion;         //用户性别 /用户地区
	private String signature;              //个性签名
	private String uid;                    //微博内部数据库用户唯一标识uid
	private String picUrl;                 //头像url
	private String userUrl;               //用户微博首页地址
	private ArrayList<String> userInfo;   //用户信息：昵称，头像url，性别/地区，关注数，粉丝数，微博数
	private Map<String,double[]> perMap = new LinkedHashMap<String,double[]>();
	
	public WeiboCrawler(String userId) throws Throwable{
		EmotionLexicon el = new EmotionLexicon();
		HashMap<String, double[]> em = el.getEmotionMap(); //情感词典读入内存中的Map
		
		NlpirInit nlpirinit = new NlpirInit();
		
		String sInput;
		String resultString;
		String postTime;
		
		String url = "http://weibo.cn/"+userId;
		String fileName = "./info/"+userId+".csv"; //perMap存储的文件名，以uid/个性域名/微号作为文件名
       
		File file = new File(fileName);
        if(file.exists())
        {
        	ObjectInputStream objectInputStream = new ObjectInputStream(new FileInputStream(file));
        	this.perMap = (LinkedHashMap<String, double[]>) objectInputStream.readObject();
        	this.userInfo = (ArrayList<String>) objectInputStream.readObject();
        	
        }
        else{
        	HashMap<String,String> cookies = WeiboCookies.read();
    		if(cookies == null){
    			try {
    				cookies = (HashMap<String, String>) WeiboCookies.getSinaCookie();
    			} catch (Exception e) {
    				// TODO Auto-generated catch block
    				e.printStackTrace();
    			}
    		}
    		
    		Connection con = Jsoup.connect(url);
    		con.userAgent("Mozilla").cookies(cookies);
    		Document doc = con.get();
    		
    		this.title = doc.title();
    		//如果打开的url标题显示“微博”或“我的首页”，说明url错误或者是当前登录用户的url，不符合要求
    		if(title.equals("微博") || title.equals("我的首页"))
    		{
    			this.idExist = false;
    			return;
    		}
    		this.screenName = title.substring(0,title.length()-3);
    		
    		statusesCountStr = doc.select("span.tc").text().replaceAll("\\D",""); //微博数字符串
    		followCount = doc.select("div.tip2 a").get(0).text().replaceAll("\\D", ""); //关注数字符串
    		fansCount = doc.select("div.tip2 a").get(1).text().replaceAll("\\D", ""); //关注数字符串
    		this.statusesCount = Integer.parseInt(statusesCountStr); //微博数字符串转换为整型
    		uid = doc.select("div.tip2 a").first().attr("href").replaceAll("\\D", ""); //获取uid
    		this.picUrl = "http://tp2.sinaimg.cn/"+uid+"/180/1";
    		this.userUrl = "http://www.weibo.com/"+uid;
    		String userInfoHtml1 = doc.select("div.ut").text();
       		
       		
       		//解决html文本中&nbsp在java中显示?问题，先将html串转义，用空格替换&nbsp;再反转义
       		userInfoHtml1 = StringEscapeUtils.escapeHtml(userInfoHtml1).replaceAll("&nbsp;", " ");
       		String userInfoStr1 = StringEscapeUtils.unescapeHtml(userInfoHtml1);
    		String[] userInfoArr1 = userInfoStr1.split("\\s");
       		this.genderAndRegion = userInfoArr1[1];
       		this.signature = userInfoArr1[5];
//    		Pattern p = Pattern.compile("^.+\\s$");
//    		Matcher mc = p.matcher(userInfoStr);
//    		while(mc.find())
//    		{
//    			mc.group();
//    		}
//    		
       		userInfo = new ArrayList();
    		userInfo.add(screenName);
    		userInfo.add(userUrl);
    		userInfo.add(picUrl);
    		userInfo.add(genderAndRegion);
    		userInfo.add(followCount);
    		userInfo.add(fansCount);
    		userInfo.add(statusesCountStr);
    		if(statusesCount%10 != 0)
    			 this.pageCount = statusesCount/10+1;
    		else 
    			 this.pageCount = statusesCount/10;
    		
    		//根据发布微博页数，循环取得发布的微博
    		for(int i=1;i<=pageCount;i++)
    		{
    			if(i>=2)
    			{
    			String urlTemp = url+"?page="+i;
    			con = Jsoup.connect(urlTemp);
    			con.userAgent("Mozilla").cookies(cookies).timeout(50000);
    		    doc = con.get();
    			}
    		    
    			Elements statuses = doc.select("div[id]").select(".c");
    			Elements postTimes = doc.select("span.ct");
    		    
    			//循环分析用户发布的每一条状态
    			for(int j=0;j<statuses.size();j++)
    			{
    				
    				String postTimeStr = postTimes.get(j).text().split("来")[0];
    		        postTime = PostTime.format(postTimeStr);
    						
    				
    				//sInput = statuses.get(j).text();
    				//sInput = statuses.get(j).select("div").last().select("span.ctt").text();
    				Element statuse = statuses.get(j);
    				Elements s = statuse.select("div").first().select("span.cmt");
    				if(s.size() != 0)
    				{
    					sInput = statuses.get(j).select("div").last().childNode(1).toString();
    				}
    				else{
    					sInput = statuses.get(j).select("div").first().select("span.ctt").text();
    				}
//    				if(sInput.startsWith("转发了"))
//    				{
//    					sInput.replaceAll("^转.*转发理由", "");
//    				}
    				resultString = nlpirinit.instance.NLPIR_ParagraphProcess(sInput, 1); // nlpir锟斤拷锟叫分达拷
    				String strEmotion;
    				double per[] = new double[7];
    				String[] strArray = resultString.split(" ");
    				
    				for (int k = 0; k < strArray.length; k++) { //遍历每一个分词
    					// System.out.println(strArray[j]);
    					
    					// 若存在匹配的表情符号，则该条微博情感值由该表情决定
    					if(strArray[k].endsWith("/xm"))
    					{
    						strEmotion = strArray[k].split("/")[0]; //取出带词性标志分词的中文词
    						if(em.containsKey(strEmotion))
    						{
    							double tempPer[] = em.get(strEmotion);
    							for(int m = 0;m<7;m++)
    							{
    								per[m] = tempPer[m];
    							}
    							break;
    						}
    								
    					}
    					
    					/*将以下有意义词性的分词取出进行分析*/
    					if (strArray[k].endsWith("/v") // 动词 5645
    							|| strArray[k].endsWith("/vi")
    							|| strArray[k].endsWith("/n")
    							|| strArray[k].endsWith("/a")
    							|| strArray[k].endsWith("/vl")
    							|| strArray[k].endsWith("/vn")
    							|| strArray[k].endsWith("/ng")
    							|| strArray[k].endsWith("/al")
    							|| strArray[k].endsWith("/an")
    							|| strArray[k].endsWith("/vf")
    							|| strArray[k].endsWith("/nl")
    							|| strArray[k].endsWith("/z")) {
    						strEmotion = strArray[k].split("/")[0]; //取出带词性标志分词的中文词
    						// System.out.println(strEmotion);
    						/*如果分词在HashMap中存在，则取出value，放入临时数组中*/
    						if (em.containsKey(strEmotion)) { 
    							double tempPer[] = em.get(strEmotion);
    							
    							int aDegree;
    							int aNegative;
    							int negNum = 0;
    							double degree = 1.0;
    							int negative = 0;
    							//查情感词所在分句前有没有程度副词，及其位置
    							for(int m=k-1;m>=0&&!strArray[m].endsWith("/wd") //逗号
    							&&!strArray[m].endsWith("/wf")//分号
    							&&!strArray[m].endsWith("/ww")//问号
    							&&!strArray[m].endsWith("/wt");m--)//感叹号
    							{
    								degree = AdverbDict.degree(strArray[m].split("/")[0]);						
    								if(degree != 1.0 ) 
    								{
    									aDegree = m;
    									break;
    								}
    							}
    							//查找情感词所在分句前有没有否定副词，及其位置和个数
    							for(int m=k-1;m>=0&&!strArray[m].endsWith("/wd") //逗号
    							&&!strArray[m].endsWith("/wf")//分号
    							&&!strArray[m].endsWith("/ww")//问号
    							&&!strArray[m].endsWith("/wt");m--){//感叹号
    								negative = AdverbDict.negative(strArray[k].split("/")[0]);
    								if(negative == -1)
    									{
    										aNegative = m;
    										negNum++;
    									}
    							}
    							/*不存在否定副词时,情感值乘上程度副词强度，再累加*/
    							if(negNum == 0)
    							for (int m = 0; m < 7; m++) {
    								per[m] += degree*tempPer[m];
    							}
    							/*存在否定副词时，且个数是奇数,厌恶和喜好，悲伤和高兴交换情感值，再累加*/
    							else if(negNum%2 == 1)
    							{
    								double temp;
    								temp = tempPer[0];
    								tempPer[0] = tempPer[4];
    								tempPer[4] = temp;
    								temp = tempPer[1];
    								tempPer[1] = tempPer[3];
    								tempPer[3] = temp;
    								
    								for (int m = 0; m < 7; m++) {
    									per[m] += degree*tempPer[m];
    								}
    							}
    							else
    								for (int m = 0; m < 7; m++) {
    									per[m] += degree*tempPer[m];
    								}
    						}
    					}

    				}
    				
    				per = Percentage.format(per);
    				perMap.put(postTime, per);
    			
    			}
    			
    		}
    		ObjectOutputStream objectOutputStream = new ObjectOutputStream(new FileOutputStream(file));
    	    objectOutputStream.writeObject(perMap);
    	    objectOutputStream.writeObject(userInfo);
    	    objectOutputStream.flush();
    	    objectOutputStream.close();
        }
		
		
	}

	public String getScreenName() {
		return screenName;
	}

	public boolean isIdExist() {
		return idExist;
	}

	public ArrayList<String> getUserInfo() {
		return userInfo;
	}

	public Map<String, double[]> getPerMap() {
		return perMap;
	}

	
}
